{
 "metadata": {
  "name": "",
  "signature": "sha256:3696fb7ffa264210dd5b8df14b41103360983cbb4835d05cf596d257e2bdbc1e"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import sys"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sys.path.append('/Users/Pro/Ongoing/weibo')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import jieba"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from weiboPredict import framework as fw"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "fw.loadData()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print '/'.join(jieba.lcut(fw.weibo_train_data.ix[0].context))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Building prefix dict from /Users/Pro/canopy/lib/python2.7/site-packages/jieba/dict.txt ...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "DEBUG:jieba:Building prefix dict from /Users/Pro/canopy/lib/python2.7/site-packages/jieba/dict.txt ...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Dumping model to file cache /var/folders/rv/scpt0ywn277bqhjg160ms0d40000gp/T/jieba.cache\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "DEBUG:jieba:Dumping model to file cache /var/folders/rv/scpt0ywn277bqhjg160ms0d40000gp/T/jieba.cache\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Loading model cost 1.978 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "DEBUG:jieba:Loading model cost 1.978 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Prefix dict has been built succesfully.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "DEBUG:jieba:Prefix dict has been built succesfully.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[/\u5410/]/@/\u4eac\u4e1c/ /\u7684/\u7528\u6237/\u4f53\u9a8c/\u592a\u5dee/\u4e86/..../\u7b2c\u4e00\u6b21/\u5728/\u4eac\u4e1c/\u8d2d\u7269/\uff0c/\u4e70/\u4e86/\u51e0\u767e/\u7684/\u4e66/..../\u771f\u662f/\u7ecf\u5386/\u4e86/\u7f51\u8d2d/\u7684/\u5404\u79cd/\u95ee\u9898/.../1/./\u8d27\u5230/\u4e86/\uff0c/\u6ca1/\u7ed9/\u53d1\u7968/\u3002/2/./\u8d27\u5230/\u4e86/\uff0c/2/\u672c\u4e66/\uff0c/\u7f51\u4e0a/\u663e\u793a/\u90fd/\u53d1\u8d27/\u4e86/\uff0c/\u5305\u88f9/\u91cc/\u53ea\u6709/1/\u672c/\uff0c/\u800c\u4e14/\u6ca1/\u53d1\u7968/.../\u95ee/\u5e97\u5bb6/\uff0c/\u5e97\u5bb6/\u627f\u8ba4/\u53e6\u5916/\u4e00\u672c/\u6ca1\u8d27/.../\u5e97\u5bb6/\u8ba9/\u6211/\u786e\u8ba4/\u6536\u8d27/\u624d/\u9000\u94b1/.../[/\u601d\u8003/]/\u3002/3/./\u6ca1\u6709/\u8d27\u7269/\u6e05\u5355/.../\u6216\u8005/\u6e05\u5355/\u662f/\u4e00\u4e2a/\u5e9f\u5f03/A4/\u6253/\u7684\n"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "type(fw.weibo_train_data.context[0])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 12,
       "text": [
        "str"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print fw.weibo_train_data.ix[0].context"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[\u5410]@\u4eac\u4e1c \u7684\u7528\u6237\u4f53\u9a8c\u592a\u5dee\u4e86....\u7b2c\u4e00\u6b21\u5728\u4eac\u4e1c\u8d2d\u7269\uff0c\u4e70\u4e86\u51e0\u767e\u7684\u4e66....\u771f\u662f\u7ecf\u5386\u4e86\u7f51\u8d2d\u7684\u5404\u79cd\u95ee\u9898...1.\u8d27\u5230\u4e86\uff0c\u6ca1\u7ed9\u53d1\u7968\u30022.\u8d27\u5230\u4e86\uff0c2\u672c\u4e66\uff0c\u7f51\u4e0a\u663e\u793a\u90fd\u53d1\u8d27\u4e86\uff0c\u5305\u88f9\u91cc\u53ea\u67091\u672c\uff0c\u800c\u4e14\u6ca1\u53d1\u7968...\u95ee\u5e97\u5bb6\uff0c\u5e97\u5bb6\u627f\u8ba4\u53e6\u5916\u4e00\u672c\u6ca1\u8d27...\u5e97\u5bb6\u8ba9\u6211\u786e\u8ba4\u6536\u8d27\u624d\u9000\u94b1...[\u601d\u8003]\u30023.\u6ca1\u6709\u8d27\u7269\u6e05\u5355...\u6216\u8005\u6e05\u5355\u662f\u4e00\u4e2a\u5e9f\u5f03A4\u6253\u7684\n"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print fw.weibo_train_data.ix[1].context"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "#\u6211\u7231\u5c0f\u7c73\u624b\u673a#\u56e0\u4e3a\u5c0f\u7c73\u624b\u673a4\u7684\u5c4f\u5e55\u592a\u7ed9\u529b\u4e86\uff0c5\u82f1\u5bf8\u7684\u9ad8\u8272\u5f69\u9971\u548c\u5ea6\u590f\u666e / JDI\u5c4f\u5e55\uff0c1920x1080\u5168\u9ad8\u6e05\u5206\u8fa8\u7387\uff0c\u73a9\u6e38\u620f\uff0c\u770b\u9ad8\u6e05\u89c6\u9891\u723d\u7206\u4e86\u300211\u670818\u65e5\u4e2d\u534812\u70b9\u5f00\u653e\u8d2d\u4e70\uff0c\u4ec5\u552e1999\u5143\uff0c\u5e0c\u671b\u80fd\u4e70\u52301\u53f0\u3002@\u5c0f\u7c73\u624b\u673a @\u5356\u4ed9\u5251\u7684\u9152\u5251\u4ed9  @\u6d77\u5148\u751fV  \uff0c\u5206\u4eab\u514d\u8d39\u9884\u7ea6\u5730\u5740http://t.cn/RPkmgBL\n"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print '/'.join('\u674e\u6668Jeremy Li is a good man\u54c8\u54c8\u54c8')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\ufffd/\ufffd/\ufffd/\ufffd/\ufffd/\ufffd/J/e/r/e/m/y/ /L/i/ /i/s/ /a/ /g/o/o/d/ /m/a/n/\ufffd/\ufffd/\ufffd/\ufffd/\ufffd/\ufffd/\ufffd/\ufffd/\ufffd\n"
       ]
      }
     ],
     "prompt_number": 60
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print fw.weibo_train_data.ix[3].context"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[doge]\u627e\u5de5\u4f5c\u256e(\u256f\u25bd\u2570)\u256d\u2026\u2026\n"
       ]
      }
     ],
     "prompt_number": 13
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print '/'.join(jieba.lcut(fw.weibo_train_data.ix[3].context))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[/doge/]/\u627e/\u5de5\u4f5c/\u256e/(/\u256f/\u25bd/\u2570/)/\u256d/\u2026/\u2026\n"
       ]
      }
     ],
     "prompt_number": 38
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print fw.weibo_train_data.ix[5].context"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "#\u9b54\u6d6a\u6f6e\u7206U5\u667a\u80fd\u8033\u673a#\u201c\u5173\u6ce8@\u9b54\u6d6a\u84dd\u7259\u8033\u673a+\u8f6c\u53d1\u672c\u5fae\u535a\u5e76@3\u4f4d\u597d\u53cb+#\u9b54\u6d6a\u6f6e\u7206U5\u667a\u80fd\u8033\u673a#+\u514d\u8d39\u4f53\u9a8c\u7684\u52a8\u4eba\u7406\u7531\u201d\u5373\u6709\u673a\u4f1a\u83b7\u5f97\u514d\u8d39\u4f53\u9a8c\u8d44\u683c\uff0c\u6211\u5df2\u7ecf\u53c2\u52a0\u4e86\uff0c\u4f60\u4eec\u5feb\u6765\u5440\uff01\uff01 http://t.cn/R7BhYdI\n"
       ]
      }
     ],
     "prompt_number": 40
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print '/'.join(jieba.lcut(fw.weibo_train_data.ix[5].context))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "#/\u9b54/\u6d6a\u6f6e/\u7206/U5/\u667a\u80fd/\u8033\u673a/#/\u201c/\u5173\u6ce8/@/\u9b54\u6d6a/\u84dd\u7259/\u8033\u673a/+/\u8f6c\u53d1/\u672c\u5fae\u535a/\u5e76/@/3/\u4f4d/\u597d\u53cb/+#/\u9b54/\u6d6a\u6f6e/\u7206/U5/\u667a\u80fd/\u8033\u673a/#+/\u514d\u8d39/\u4f53\u9a8c/\u7684/\u52a8\u4eba/\u7406\u7531/\u201d/\u5373/\u6709/\u673a\u4f1a/\u83b7\u5f97/\u514d\u8d39/\u4f53\u9a8c/\u8d44\u683c/\uff0c/\u6211/\u5df2\u7ecf/\u53c2\u52a0/\u4e86/\uff0c/\u4f60\u4eec/\u5feb\u6765/\u5440/\uff01/\uff01/ /http/://///t/./cn///R7BhYdI\n"
       ]
      }
     ],
     "prompt_number": 42
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "1. \u8868\u60c5\u7b26\u53f7\uff08\u5305\u62ecemojo\u548c\u6807\u70b9\u7b26\u53f7\u7ec4\u6210\u7684\u8868\u60c5\uff09-> \u6682\u65f6\u4f5c\u4e3a\u6b63\u5e38\u503c\u5206\u8bcd\n",
      "2. @\u67d0\u4f4d\u7528\u6237 -> \u76ee\u524d\u6307\u53bb\u9664@\uff0c\u7528\u6237\u540d\u6b63\u5e38\u5206\u8bcd\n",
      "3. \u6807\u7b7e## -> \u76ee\u524d\u5148\u53ea\u53bb\u9664#\uff0c\u6807\u7b7e\u6b63\u5e38\u5206\u8bcd\n",
      "4. \u94fe\u63a5 -> \u6b63\u5219\u8868\u8fbe\u5f0f\u53bb\u9664\n",
      "5. \u505c\u7528\u8bcd -> \u505c\u7528\u8bcd\u8868\u53bb\u9664\n",
      "6. \u6570\u5b57 -> \u6b63\u5219\u8868\u8fbe\u5f0f\u53bb\u9664\n",
      "7. \u6807\u70b9\u7b26\u53f7 -> \u6682\u65f6\u5305\u62ec\u5728\u505c\u7528\u8bcd\u4e2d\u53bb\u9664\n",
      "8. \u7a7a\u683c -> \u53bb\u9664\n",
      "9. \u82f1\u6587 -> \u6682\u65f6\u4e0d\u53bb\u9664\n",
      "10. \u65f6\u95f4\u65e5\u671f -> \u6682\u65f6\u4e0d\u5904\u7406"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print '/'.join(jieba.lcut(\"my nameeeeeeeee is lichen\"))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "my/ /nameeeeeeeee/ /is/ /lichen\n"
       ]
      }
     ],
     "prompt_number": 45
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for c in fw.weibo_train_data.context[:5]:\n",
      "    print c"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[\u5410]@\u4eac\u4e1c \u7684\u7528\u6237\u4f53\u9a8c\u592a\u5dee\u4e86....\u7b2c\u4e00\u6b21\u5728\u4eac\u4e1c\u8d2d\u7269\uff0c\u4e70\u4e86\u51e0\u767e\u7684\u4e66....\u771f\u662f\u7ecf\u5386\u4e86\u7f51\u8d2d\u7684\u5404\u79cd\u95ee\u9898...1.\u8d27\u5230\u4e86\uff0c\u6ca1\u7ed9\u53d1\u7968\u30022.\u8d27\u5230\u4e86\uff0c2\u672c\u4e66\uff0c\u7f51\u4e0a\u663e\u793a\u90fd\u53d1\u8d27\u4e86\uff0c\u5305\u88f9\u91cc\u53ea\u67091\u672c\uff0c\u800c\u4e14\u6ca1\u53d1\u7968...\u95ee\u5e97\u5bb6\uff0c\u5e97\u5bb6\u627f\u8ba4\u53e6\u5916\u4e00\u672c\u6ca1\u8d27...\u5e97\u5bb6\u8ba9\u6211\u786e\u8ba4\u6536\u8d27\u624d\u9000\u94b1...[\u601d\u8003]\u30023.\u6ca1\u6709\u8d27\u7269\u6e05\u5355...\u6216\u8005\u6e05\u5355\u662f\u4e00\u4e2a\u5e9f\u5f03A4\u6253\u7684\n",
        "#\u6211\u7231\u5c0f\u7c73\u624b\u673a#\u56e0\u4e3a\u5c0f\u7c73\u624b\u673a4\u7684\u5c4f\u5e55\u592a\u7ed9\u529b\u4e86\uff0c5\u82f1\u5bf8\u7684\u9ad8\u8272\u5f69\u9971\u548c\u5ea6\u590f\u666e / JDI\u5c4f\u5e55\uff0c1920x1080\u5168\u9ad8\u6e05\u5206\u8fa8\u7387\uff0c\u73a9\u6e38\u620f\uff0c\u770b\u9ad8\u6e05\u89c6\u9891\u723d\u7206\u4e86\u300211\u670818\u65e5\u4e2d\u534812\u70b9\u5f00\u653e\u8d2d\u4e70\uff0c\u4ec5\u552e1999\u5143\uff0c\u5e0c\u671b\u80fd\u4e70\u52301\u53f0\u3002@\u5c0f\u7c73\u624b\u673a @\u5356\u4ed9\u5251\u7684\u9152\u5251\u4ed9  @\u6d77\u5148\u751fV  \uff0c\u5206\u4eab\u514d\u8d39\u9884\u7ea6\u5730\u5740http://t.cn/RPkmgBL\n",
        "[doge][\u55b5\u55b5]\u5929\u732b\u771f\u662f\u725b....\u5356\u5b9e\u9a8c\u8017\u6750\u7684\u90fd\u8dd1\u5929\u732b\u53bb\u5f00\u5e97\u4e86...\n",
        "[doge]\u627e\u5de5\u4f5c\u256e(\u256f\u25bd\u2570)\u256d\u2026\u2026\n",
        "[doge]\u6210\u529f\u62a2\u4e86\u53f0\u7c734\u7535\u4fe1~\n"
       ]
      }
     ],
     "prompt_number": 48
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "f=open('../data/stopwords.txt','r')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 89
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "stopwords = [l.strip() for l in f.readlines()]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 90
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print stopwords[100]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\u800c\u5916\n"
       ]
      }
     ],
     "prompt_number": 18
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "text = jieba.lcut(fw.weibo_train_data.context[0])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 152
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "clean = [t for t in text if t not in stopwords]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 170
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "stopwords[2].encode('utf8')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "UnicodeDecodeError",
       "evalue": "'ascii' codec can't decode byte 0xe3 in position 0: ordinal not in range(128)",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mUnicodeDecodeError\u001b[0m                        Traceback (most recent call last)",
        "\u001b[0;32m<ipython-input-20-995d90a93e63>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mstopwords\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'utf8'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
        "\u001b[0;31mUnicodeDecodeError\u001b[0m: 'ascii' codec can't decode byte 0xe3 in position 0: ordinal not in range(128)"
       ]
      }
     ],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for i in range(len(stopwords)):\n",
      "    stopwords[i] = stopwords[i].decode('utf8')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 91
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print stopwords[100]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\u800c\u5916\n"
       ]
      }
     ],
     "prompt_number": 78
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print '/'.join(clean)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\u5410/\u4eac\u4e1c/ /\u7528\u6237/\u4f53\u9a8c/\u592a\u5dee/..../\u7b2c\u4e00\u6b21/\u4eac\u4e1c/\u8d2d\u7269/\u4e70/\u51e0\u767e/\u4e66/..../\u771f\u662f/\u7ecf\u5386/\u7f51\u8d2d/\u95ee\u9898/\u8d27\u5230/\u53d1\u7968/\u8d27\u5230/\u672c\u4e66/\u7f51\u4e0a/\u663e\u793a/\u53d1\u8d27/\u5305\u88f9/\u91cc/\u53d1\u7968/\u95ee/\u5e97\u5bb6/\u5e97\u5bb6/\u627f\u8ba4/\u4e00\u672c/\u6ca1\u8d27/\u5e97\u5bb6/\u786e\u8ba4/\u6536\u8d27/\u9000\u94b1/\u601d\u8003/\u8d27\u7269/\u6e05\u5355/\u6e05\u5355/\u4e00\u4e2a/\u5e9f\u5f03/A4\n"
       ]
      }
     ],
     "prompt_number": 171
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print '/'.join(jieba.lcut(fw.weibo_train_data.ix[0].context))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[/\u5410/]/@/\u4eac\u4e1c/ /\u7684/\u7528\u6237/\u4f53\u9a8c/\u592a\u5dee/\u4e86/..../\u7b2c\u4e00\u6b21/\u5728/\u4eac\u4e1c/\u8d2d\u7269/\uff0c/\u4e70/\u4e86/\u51e0\u767e/\u7684/\u4e66/..../\u771f\u662f/\u7ecf\u5386/\u4e86/\u7f51\u8d2d/\u7684/\u5404\u79cd/\u95ee\u9898/.../1/./\u8d27\u5230/\u4e86/\uff0c/\u6ca1/\u7ed9/\u53d1\u7968/\u3002/2/./\u8d27\u5230/\u4e86/\uff0c/2/\u672c\u4e66/\uff0c/\u7f51\u4e0a/\u663e\u793a/\u90fd/\u53d1\u8d27/\u4e86/\uff0c/\u5305\u88f9/\u91cc/\u53ea\u6709/1/\u672c/\uff0c/\u800c\u4e14/\u6ca1/\u53d1\u7968/.../\u95ee/\u5e97\u5bb6/\uff0c/\u5e97\u5bb6/\u627f\u8ba4/\u53e6\u5916/\u4e00\u672c/\u6ca1\u8d27/.../\u5e97\u5bb6/\u8ba9/\u6211/\u786e\u8ba4/\u6536\u8d27/\u624d/\u9000\u94b1/.../[/\u601d\u8003/]/\u3002/3/./\u6ca1\u6709/\u8d27\u7269/\u6e05\u5355/.../\u6216\u8005/\u6e05\u5355/\u662f/\u4e00\u4e2a/\u5e9f\u5f03/A4/\u6253/\u7684\n"
       ]
      }
     ],
     "prompt_number": 138
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "text[10] in stopwords"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 169,
       "text": [
        "True"
       ]
      }
     ],
     "prompt_number": 169
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "'1ST\u91cc'.lower()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 44,
       "text": [
        "'1st\\xe9\\x87\\x8c'"
       ]
      }
     ],
     "prompt_number": 44
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print stopwords[212]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\u4e86\n"
       ]
      }
     ],
     "prompt_number": 29
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print re.findall(\"http://.*\\w$\",fw.weibo_train_data.context[9])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[]\n"
       ]
      }
     ],
     "prompt_number": 62
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for c in fw.weibo_train_data.context[:20]:\n",
      "    print c"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[\u5410]@\u4eac\u4e1c \u7684\u7528\u6237\u4f53\u9a8c\u592a\u5dee\u4e86....\u7b2c\u4e00\u6b21\u5728\u4eac\u4e1c\u8d2d\u7269\uff0c\u4e70\u4e86\u51e0\u767e\u7684\u4e66....\u771f\u662f\u7ecf\u5386\u4e86\u7f51\u8d2d\u7684\u5404\u79cd\u95ee\u9898...1.\u8d27\u5230\u4e86\uff0c\u6ca1\u7ed9\u53d1\u7968\u30022.\u8d27\u5230\u4e86\uff0c2\u672c\u4e66\uff0c\u7f51\u4e0a\u663e\u793a\u90fd\u53d1\u8d27\u4e86\uff0c\u5305\u88f9\u91cc\u53ea\u67091\u672c\uff0c\u800c\u4e14\u6ca1\u53d1\u7968...\u95ee\u5e97\u5bb6\uff0c\u5e97\u5bb6\u627f\u8ba4\u53e6\u5916\u4e00\u672c\u6ca1\u8d27...\u5e97\u5bb6\u8ba9\u6211\u786e\u8ba4\u6536\u8d27\u624d\u9000\u94b1...[\u601d\u8003]\u30023.\u6ca1\u6709\u8d27\u7269\u6e05\u5355...\u6216\u8005\u6e05\u5355\u662f\u4e00\u4e2a\u5e9f\u5f03A4\u6253\u7684\n",
        "#\u6211\u7231\u5c0f\u7c73\u624b\u673a#\u56e0\u4e3a\u5c0f\u7c73\u624b\u673a4\u7684\u5c4f\u5e55\u592a\u7ed9\u529b\u4e86\uff0c5\u82f1\u5bf8\u7684\u9ad8\u8272\u5f69\u9971\u548c\u5ea6\u590f\u666e / JDI\u5c4f\u5e55\uff0c1920x1080\u5168\u9ad8\u6e05\u5206\u8fa8\u7387\uff0c\u73a9\u6e38\u620f\uff0c\u770b\u9ad8\u6e05\u89c6\u9891\u723d\u7206\u4e86\u300211\u670818\u65e5\u4e2d\u534812\u70b9\u5f00\u653e\u8d2d\u4e70\uff0c\u4ec5\u552e1999\u5143\uff0c\u5e0c\u671b\u80fd\u4e70\u52301\u53f0\u3002@\u5c0f\u7c73\u624b\u673a @\u5356\u4ed9\u5251\u7684\u9152\u5251\u4ed9  @\u6d77\u5148\u751fV  \uff0c\u5206\u4eab\u514d\u8d39\u9884\u7ea6\u5730\u5740http://t.cn/RPkmgBL\n",
        "[doge][\u55b5\u55b5]\u5929\u732b\u771f\u662f\u725b....\u5356\u5b9e\u9a8c\u8017\u6750\u7684\u90fd\u8dd1\u5929\u732b\u53bb\u5f00\u5e97\u4e86...\n",
        "[doge]\u627e\u5de5\u4f5c\u256e(\u256f\u25bd\u2570)\u256d\u2026\u2026\n",
        "[doge]\u6210\u529f\u62a2\u4e86\u53f0\u7c734\u7535\u4fe1~\n",
        "#\u9b54\u6d6a\u6f6e\u7206U5\u667a\u80fd\u8033\u673a#\u201c\u5173\u6ce8@\u9b54\u6d6a\u84dd\u7259\u8033\u673a+\u8f6c\u53d1\u672c\u5fae\u535a\u5e76@3\u4f4d\u597d\u53cb+#\u9b54\u6d6a\u6f6e\u7206U5\u667a\u80fd\u8033\u673a#+\u514d\u8d39\u4f53\u9a8c\u7684\u52a8\u4eba\u7406\u7531\u201d\u5373\u6709\u673a\u4f1a\u83b7\u5f97\u514d\u8d39\u4f53\u9a8c\u8d44\u683c\uff0c\u6211\u5df2\u7ecf\u53c2\u52a0\u4e86\uff0c\u4f60\u4eec\u5feb\u6765\u5440\uff01\uff01 http://t.cn/R7BhYdI\n",
        "[\u53ef\u601c] \u53f2\u4e0a\u6700\u96be\u767b\u7684\u7684\u7f51\u7ad9\uff1ahttp://t.cn/R78fdMf\n",
        "[\u6253\u8138]\u524d\u5929\u786e\u8ba4\u62a5\u540d\u3002\u3002\u3002\u3002\u4eca\u5929\u5356\u72ec\u5bb6\u5185\u90e8\u8d44\u6599\u7684\u7535\u8bdd\u5c31\u6765\u4e86\u3002\u3002\u3002\u3002\n",
        "\u53cc\u5341\u4e00\u540e\u9057\u75c7....42\u7801\u677f\u978b...\u5168\u7f3a\u8d27...\n",
        "\u3010\u8bba\u575b\u6e90\u7801\u63a8\u8350\uff086\u670830\u65e5\uff09\uff1aiOS\u5f00\u6e90RSS\u65b0\u95fb\u9605\u8bfb\u5668 \u53ef\u81ea\u52a8\u586b\u5145\u7684\u6587\u672c\u6846\u3011 http://t.cn/RvuICAU  \uff08\u5206\u4eab\u81ea@\u65e0\u89c5\u9605\u8bfb\uff09\n",
        "\u3010Hadoop\u79d1\u666e\u6587\u2014\u2014\u5e38\u89c1\u768445\u4e2a\u95ee\u9898\u89e3\u7b54\u3011 http://t.cn/Rvrui7S  \uff08\u5206\u4eab\u81ea@\u65e0\u89c5\u9605\u8bfb\uff09\n",
        "\u3010MySQL\u7d22\u5f15\u539f\u7406\u53ca\u6162\u67e5\u8be2\u4f18\u5316\u3011 http://t.cn/RvsXH5D  \uff08\u5206\u4eab\u81ea@\u65e0\u89c5\u9605\u8bfb\uff09\n",
        "\u3010\u7f51\u7ad9\u7528\u6237\u8bbf\u95ee\u901f\u5ea6\u76d1\u6d4b\u5206\u6790\u9879\u76ee(1)\u3011 http://t.cn/RPLC0uY  \uff08\u5206\u4eab\u81ea@\u65e0\u89c5\u9605\u8bfb\uff09\n",
        "\u3010\u4e0d\u540cvlan\u95f4\u901a\u4fe1\u5982\u4f55\u5b9e\u73b0\uff1f\u3011 http://t.cn/RPLOZ49  \uff08\u5206\u4eab\u81ea@\u65e0\u89c5\u9605\u8bfb\uff09\n",
        "\u5c0f\u4f19\u4f34\u4eec\uff0c\u5feb\u6765\u56f4\u89c2\uff01\u6211\u5df2\u7ecf\u5347\u7ea7\u4e3aV6\u65b0\u7248\u5fae\u535a\uff0c\u7b80\u6d01\u7684\u754c\u9762\u5e26\u6765\u66f4\u6d41\u7545\u7684\u4f53\u9a8c\u3002\u51c6\u5907\u597d\u4e86\u5417\uff1f\u548c\u6211\u4eec\u4e00\u8d77\u53d1\u73b0\u65b0\u7684\u4e16\u754c\u5427\uff01\u5347\u7ea7\u731b\u6233:http://t.cn/R7vgnNI http://t.cn/R7PW0Ng\n",
        "\u6211\u521a\u6362\u4e86\u201c\u60c5\u8ff7\u5b9d\u4e3d\u6765\u201d\u5957\u88c5\uff0c\u597d\u6f02\u4eae\uff0c\u4f60\u4eec\u90fd\u5feb\u6765\u8bd5\u8bd5\uff01 http://weibo.com/home?skinId=skin053\n",
        "5k \u7684iMac\u597d\u50cf\u5f88\u4e0d\u9519\n",
        "\u4eca\u5929\u9047\u5230\u4e86\u4e24\u4e2a\u5947\u8469\n",
        "\u6ca1\u4e8b\uff0c\u53ea\u662f\u6280\u672f\u8c03\u6574\n",
        "\u81ea\u5df1\u628a\u8138\u90fd\u6293\u4f24\u4e86\uff0c\u6240\u4ee5\u5e26\u4e0a\u4e86\u624b\u5957\u2026 http://t.cn/Rv3EAY4\n"
       ]
      }
     ],
     "prompt_number": 103
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import re\n",
      "from nltk.corpus import stopwords as e_stopwords\n",
      "for i,context in enumerate(fw.weibo_train_data.context[0:20]):\n",
      "    print str(i)+'.'\n",
      "    print context\n",
      "    #\u53bb\u9664\u94fe\u63a5\n",
      "    context = re.sub(\"http://.*\\w\",\"\",context)\n",
      "    #\u53bb\u9664\u8868\u60c5\u7b26\u53f7\n",
      "    context = re.sub(\"\\[.{0,12}\\]\",\"\",context)\n",
      "    #\u53bb\u9664\u6807\u70b9\u7b26\u53f7\n",
      "    #context = re.sub(\"\\\\pP|\\\\pS\", \"\",context)\n",
      "    context = re.sub(\"[\\s+\\.\\!\\/_,$%^*(+\\\"\\']+|[+\u2014\u2014\uff01\uff0c\u3002\uff1f\u3001~@#\uffe5%\u2026\u2026&*\uff08\uff09\u3010\u3011\u256e\u256f\u25bd\u2570\u256d]+\".decode(\"utf8\"), \"\",context)\n",
      "    #\u82f1\u6587\u5b57\u6bcd\u53d8\u5c0f\u5199\n",
      "    context  = context.lower()\n",
      "    #\u627e\u51fa\u6240\u6709\u82f1\u6587\u5b57\u7b26\u548c\u5355\u8bcd\u5e76\u63d0\u53d6\n",
      "    english = re.findall(\"[a-z]+\",context)\n",
      "    context = re.sub(\"[a-z]+\",\"\",context)\n",
      "    #\u53bb\u9664\u7a7a\u683c\n",
      "    context = re.sub(\"\\s\",\"\",context)\n",
      "    #\u53bb\u9664\u6240\u6709\u6570\u5b57\n",
      "    context = re.sub(\"\\d\",\"\",context)\n",
      "    #\u53bb\u9664\u81ea\u5df1\u6253\u7684\u7701\u7565\u53f7\n",
      "    context = re.sub(\"\\.*\",\"\",context)\n",
      "    #\u5206\u8bcd\n",
      "    text = jieba.lcut(context)\n",
      "    #\u53bb\u9664\u4e2d\u6587\u505c\u7528\u8bcd\n",
      "    clean = [t for t in text if t not in stopwords]\n",
      "    #\u53bb\u9664\u82f1\u6587\u505c\u7528\u8bcd\u4e0e\u5355\u4e2a\u82f1\u6587\u5b57\u6bcd\n",
      "    e_clean = [t for t in english if t not in e_stopwords.words('english') and len(t) is not 1]\n",
      "    clean+=e_clean\n",
      "    \n",
      "    print '|'.join(clean)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0.\n",
        "[\u5410]@\u4eac\u4e1c \u7684\u7528\u6237\u4f53\u9a8c\u592a\u5dee\u4e86....\u7b2c\u4e00\u6b21\u5728\u4eac\u4e1c\u8d2d\u7269\uff0c\u4e70\u4e86\u51e0\u767e\u7684\u4e66....\u771f\u662f\u7ecf\u5386\u4e86\u7f51\u8d2d\u7684\u5404\u79cd\u95ee\u9898...1.\u8d27\u5230\u4e86\uff0c\u6ca1\u7ed9\u53d1\u7968\u30022.\u8d27\u5230\u4e86\uff0c2\u672c\u4e66\uff0c\u7f51\u4e0a\u663e\u793a\u90fd\u53d1\u8d27\u4e86\uff0c\u5305\u88f9\u91cc\u53ea\u67091\u672c\uff0c\u800c\u4e14\u6ca1\u53d1\u7968...\u95ee\u5e97\u5bb6\uff0c\u5e97\u5bb6\u627f\u8ba4\u53e6\u5916\u4e00\u672c\u6ca1\u8d27...\u5e97\u5bb6\u8ba9\u6211\u786e\u8ba4\u6536\u8d27\u624d\u9000\u94b1...[\u601d\u8003]\u30023.\u6ca1\u6709\u8d27\u7269\u6e05\u5355...\u6216\u8005\u6e05\u5355\u662f\u4e00\u4e2a\u5e9f\u5f03A4\u6253\u7684\n",
        "\u4eac\u4e1c|\u7528\u6237|\u4f53\u9a8c|\u592a\u5dee|\u7b2c\u4e00\u6b21|\u4eac\u4e1c|\u8d2d\u7269|\u4e70|\u51e0\u767e|\u4e66|\u771f\u662f|\u7ecf\u5386|\u7f51\u8d2d|\u95ee\u9898|\u8d27\u5230|\u53d1\u7968|\u8d27\u5230|\u672c\u4e66|\u7f51\u4e0a|\u663e\u793a|\u53d1\u8d27|\u5305\u88f9|\u91cc|\u53d1\u7968|\u95ee|\u5e97\u5bb6|\u5e97\u5bb6|\u627f\u8ba4|\u4e00\u672c|\u6ca1\u8d27|\u5e97\u5bb6|\u786e\u8ba4|\u6536\u8d27|\u9000\u94b1|\u8d27\u7269|\u6e05\u5355|\u6e05\u5355|\u4e00\u4e2a|\u5e9f\u5f03\n",
        "1.\n",
        "#\u6211\u7231\u5c0f\u7c73\u624b\u673a#\u56e0\u4e3a\u5c0f\u7c73\u624b\u673a4\u7684\u5c4f\u5e55\u592a\u7ed9\u529b\u4e86\uff0c5\u82f1\u5bf8\u7684\u9ad8\u8272\u5f69\u9971\u548c\u5ea6\u590f\u666e / JDI\u5c4f\u5e55\uff0c1920x1080\u5168\u9ad8\u6e05\u5206\u8fa8\u7387\uff0c\u73a9\u6e38\u620f\uff0c\u770b\u9ad8\u6e05\u89c6\u9891\u723d\u7206\u4e86\u300211\u670818\u65e5\u4e2d\u534812\u70b9\u5f00\u653e\u8d2d\u4e70\uff0c\u4ec5\u552e1999\u5143\uff0c\u5e0c\u671b\u80fd\u4e70\u52301\u53f0\u3002@\u5c0f\u7c73\u624b\u673a @\u5356\u4ed9\u5251\u7684\u9152\u5251\u4ed9  @\u6d77\u5148\u751fV  \uff0c\u5206\u4eab\u514d\u8d39\u9884\u7ea6\u5730\u5740http://t.cn/RPkmgBL\n",
        "\u7231|\u5c0f\u7c73|\u624b\u673a|\u5c0f\u7c73|\u624b\u673a|\u5c4f\u5e55|\u592a\u7ed9\u529b|\u82f1\u5bf8|\u9ad8|\u8272\u5f69|\u9971\u548c\u5ea6|\u590f\u666e|\u5c4f\u5e55|\u5168\u9ad8\u6e05|\u5206\u8fa8\u7387|\u73a9\u6e38\u620f|\u9ad8\u6e05|\u89c6\u9891|\u723d\u7206|\u4e2d\u5348|\u70b9|\u5f00\u653e|\u8d2d\u4e70|\u4ec5\u552e\u5143|\u5e0c\u671b|\u4e70|\u53f0|\u5c0f\u7c73|\u624b\u673a|\u5356|\u4ed9\u5251|\u9152\u5251\u4ed9|\u6d77|\u5148\u751f|\u5206\u4eab|\u514d\u8d39|\u9884\u7ea6|\u5730\u5740|jdi\n",
        "2.\n",
        "[doge][\u55b5\u55b5]\u5929\u732b\u771f\u662f\u725b....\u5356\u5b9e\u9a8c\u8017\u6750\u7684\u90fd\u8dd1\u5929\u732b\u53bb\u5f00\u5e97\u4e86...\n",
        "\u5929\u732b|\u771f\u662f|\u725b\u5356|\u5b9e\u9a8c|\u8017\u6750|\u8dd1|\u5929\u732b|\u5f00\u5e97\n",
        "3.\n",
        "[doge]\u627e\u5de5\u4f5c\u256e(\u256f\u25bd\u2570)\u256d\u2026\u2026\n",
        "\u627e|\u5de5\u4f5c|\u256e|\u256f|\u25bd|\u2570|\u256d\n",
        "4.\n",
        "[doge]\u6210\u529f\u62a2\u4e86\u53f0\u7c734\u7535\u4fe1~\n",
        "\u6210\u529f|\u62a2|\u53f0\u7c73|\u7535\u4fe1\n",
        "5.\n",
        "#\u9b54\u6d6a\u6f6e\u7206U5\u667a\u80fd\u8033\u673a#\u201c\u5173\u6ce8@\u9b54\u6d6a\u84dd\u7259\u8033\u673a+\u8f6c\u53d1\u672c\u5fae\u535a\u5e76@3\u4f4d\u597d\u53cb+#\u9b54\u6d6a\u6f6e\u7206U5\u667a\u80fd\u8033\u673a#+\u514d\u8d39\u4f53\u9a8c\u7684\u52a8\u4eba\u7406\u7531\u201d\u5373\u6709\u673a\u4f1a\u83b7\u5f97\u514d\u8d39\u4f53\u9a8c\u8d44\u683c\uff0c\u6211\u5df2\u7ecf\u53c2\u52a0\u4e86\uff0c\u4f60\u4eec\u5feb\u6765\u5440\uff01\uff01 http://t.cn/R7BhYdI\n",
        "\u9b54|\u6d6a\u6f6e|\u7206|\u667a\u80fd|\u8033\u673a|\u5173\u6ce8|\u9b54\u6d6a|\u84dd\u7259|\u8033\u673a|\u8f6c\u53d1|\u672c\u5fae\u535a|\u5e76\u4f4d|\u597d\u53cb|\u9b54|\u6d6a\u6f6e|\u7206|\u667a\u80fd|\u8033\u673a|\u514d\u8d39|\u4f53\u9a8c|\u52a8\u4eba|\u7406\u7531|\u673a\u4f1a|\u83b7\u5f97|\u514d\u8d39|\u4f53\u9a8c|\u8d44\u683c|\u5df2\u7ecf|\u53c2\u52a0|\u5feb\u6765\n",
        "6.\n",
        "[\u53ef\u601c] \u53f2\u4e0a\u6700\u96be\u767b\u7684\u7684\u7f51\u7ad9\uff1ahttp://t.cn/R78fdMf\n",
        "\u53f2\u4e0a|\u6700\u96be|\u767b|\u7f51\u7ad9\n",
        "7.\n",
        "[\u6253\u8138]\u524d\u5929\u786e\u8ba4\u62a5\u540d\u3002\u3002\u3002\u3002\u4eca\u5929\u5356\u72ec\u5bb6\u5185\u90e8\u8d44\u6599\u7684\u7535\u8bdd\u5c31\u6765\u4e86\u3002\u3002\u3002\u3002\n",
        "\u524d\u5929|\u786e\u8ba4|\u62a5\u540d|\u4eca\u5929|\u5356|\u72ec\u5bb6|\u5185\u90e8\u8d44\u6599|\u7535\u8bdd\n",
        "8.\n",
        "\u53cc\u5341\u4e00\u540e\u9057\u75c7....42\u7801\u677f\u978b...\u5168\u7f3a\u8d27...\n",
        "\u53cc\u5341|\u540e\u9057\u75c7|\u7801\u677f|\u978b\u5168|\u7f3a\u8d27\n",
        "9.\n",
        "\u3010\u8bba\u575b\u6e90\u7801\u63a8\u8350\uff086\u670830\u65e5\uff09\uff1aiOS\u5f00\u6e90RSS\u65b0\u95fb\u9605\u8bfb\u5668 \u53ef\u81ea\u52a8\u586b\u5145\u7684\u6587\u672c\u6846\u3011 http://t.cn/RvuICAU  \uff08\u5206\u4eab\u81ea@\u65e0\u89c5\u9605\u8bfb\uff09\n",
        "\u3010|\u8bba\u575b|\u6e90\u7801|\u63a8\u8350|\u5f00\u6e90|\u65b0\u95fb|\u9605\u8bfb\u5668|\u81ea\u52a8|\u586b\u5145|\u6587\u672c\u6846|\u3011|\u5206\u4eab|\u81ea\u65e0\u89c5|\u9605\u8bfb|ios|rss\n",
        "10.\n",
        "\u3010Hadoop\u79d1\u666e\u6587\u2014\u2014\u5e38\u89c1\u768445\u4e2a\u95ee\u9898\u89e3\u7b54\u3011 http://t.cn/Rvrui7S  \uff08\u5206\u4eab\u81ea@\u65e0\u89c5\u9605\u8bfb\uff09\n",
        "\u3010|\u79d1\u666e|\u6587|\u5e38\u89c1|\u95ee\u9898\u89e3\u7b54|\u3011|\u5206\u4eab|\u81ea\u65e0\u89c5|\u9605\u8bfb|hadoop\n",
        "11.\n",
        "\u3010MySQL\u7d22\u5f15\u539f\u7406\u53ca\u6162\u67e5\u8be2\u4f18\u5316\u3011 http://t.cn/RvsXH5D  \uff08\u5206\u4eab\u81ea@\u65e0\u89c5\u9605\u8bfb\uff09\n",
        "\u3010|\u7d22\u5f15|\u539f\u7406|\u53ca\u6162|\u67e5\u8be2|\u4f18\u5316|\u3011|\u5206\u4eab|\u81ea\u65e0\u89c5|\u9605\u8bfb|mysql\n",
        "12.\n",
        "\u3010\u7f51\u7ad9\u7528\u6237\u8bbf\u95ee\u901f\u5ea6\u76d1\u6d4b\u5206\u6790\u9879\u76ee(1)\u3011 http://t.cn/RPLC0uY  \uff08\u5206\u4eab\u81ea@\u65e0\u89c5\u9605\u8bfb\uff09\n",
        "\u3010|\u7f51\u7ad9|\u7528\u6237|\u8bbf\u95ee\u901f\u5ea6|\u76d1\u6d4b|\u5206\u6790|\u9879\u76ee|\u3011|\u5206\u4eab|\u81ea\u65e0\u89c5|\u9605\u8bfb\n",
        "13.\n",
        "\u3010\u4e0d\u540cvlan\u95f4\u901a\u4fe1\u5982\u4f55\u5b9e\u73b0\uff1f\u3011 http://t.cn/RPLOZ49  \uff08\u5206\u4eab\u81ea@\u65e0\u89c5\u9605\u8bfb\uff09\n",
        "\u3010|\u95f4\u901a\u4fe1|\u5b9e\u73b0|\u3011|\u5206\u4eab|\u81ea\u65e0\u89c5|\u9605\u8bfb|vlan\n",
        "14.\n",
        "\u5c0f\u4f19\u4f34\u4eec\uff0c\u5feb\u6765\u56f4\u89c2\uff01\u6211\u5df2\u7ecf\u5347\u7ea7\u4e3aV6\u65b0\u7248\u5fae\u535a\uff0c\u7b80\u6d01\u7684\u754c\u9762\u5e26\u6765\u66f4\u6d41\u7545\u7684\u4f53\u9a8c\u3002\u51c6\u5907\u597d\u4e86\u5417\uff1f\u548c\u6211\u4eec\u4e00\u8d77\u53d1\u73b0\u65b0\u7684\u4e16\u754c\u5427\uff01\u5347\u7ea7\u731b\u6233:http://t.cn/R7vgnNI http://t.cn/R7PW0Ng\n",
        "\u5c0f\u4f19\u4f34|\u56f4\u89c2|\u5df2\u7ecf|\u5347\u7ea7|\u65b0\u7248|\u5fae\u535a|\u7b80\u6d01|\u754c\u9762|\u5e26\u6765|\u6d41\u7545|\u4f53\u9a8c|\u51c6\u5907|\u597d\u4e86\u5417|\u4e00\u8d77|\u53d1\u73b0|\u65b0|\u4e16\u754c|\u5347\u7ea7|\u731b|\u6233\n",
        "15.\n",
        "\u6211\u521a\u6362\u4e86\u201c\u60c5\u8ff7\u5b9d\u4e3d\u6765\u201d\u5957\u88c5\uff0c\u597d\u6f02\u4eae\uff0c\u4f60\u4eec\u90fd\u5feb\u6765\u8bd5\u8bd5\uff01 http://weibo.com/home?skinId=skin053\n",
        "\u521a\u6362|\u60c5\u8ff7|\u5b9d\u4e3d\u6765|\u5957\u88c5|\u597d|\u6f02\u4eae|\u8bd5\u8bd5\n",
        "16.\n",
        "5k \u7684iMac\u597d\u50cf\u5f88\u4e0d\u9519\n",
        "\u597d\u50cf|\u4e0d\u9519|imac\n",
        "17.\n",
        "\u4eca\u5929\u9047\u5230\u4e86\u4e24\u4e2a\u5947\u8469\n",
        "\u4eca\u5929|\u9047\u5230|\u4e24\u4e2a|\u5947\u8469\n",
        "18.\n",
        "\u6ca1\u4e8b\uff0c\u53ea\u662f\u6280\u672f\u8c03\u6574\n",
        "\u6ca1\u4e8b|\u6280\u672f|\u8c03\u6574\n",
        "19.\n",
        "\u81ea\u5df1\u628a\u8138\u90fd\u6293\u4f24\u4e86\uff0c\u6240\u4ee5\u5e26\u4e0a\u4e86\u624b\u5957\u2026 http://t.cn/Rv3EAY4\n",
        "\u8138|\u6293\u4f24|\u5e26\u4e0a|\u624b\u5957\n"
       ]
      }
     ],
     "prompt_number": 122
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print re.findall('[\\s+\\.\\!\\/_,$%^*(+\\\"\\']+|[+\u2014\u2014\uff01\uff0c\u3002\uff1f\u3001~@#\uffe5%\u2026\u2026&*\uff08\uff09\u3010\u3011\u256e\u256f\u25bd\u2570\u256d]+',fw.weibo_train_data.context[9])[1].decode('utf8')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "UnicodeDecodeError",
       "evalue": "'utf8' codec can't decode byte 0xae in position 0: invalid start byte",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mUnicodeDecodeError\u001b[0m                        Traceback (most recent call last)",
        "\u001b[0;32m<ipython-input-121-84060324dc73>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mprint\u001b[0m \u001b[0mre\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfindall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'[\\s+\\.\\!\\/_,$%^*(+\\\"\\']+|[+\u2014\u2014\uff01\uff0c\u3002\uff1f\u3001~@#\uffe5%\u2026\u2026&*\uff08\uff09\u3010\u3011\u256e\u256f\u25bd\u2570\u256d]+'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfw\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweibo_train_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontext\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m9\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'utf8'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
        "\u001b[0;32m/Applications/Canopy.app/appdata/canopy-1.4.1.1975.macosx-x86_64/Canopy.app/Contents/lib/python2.7/encodings/utf_8.pyc\u001b[0m in \u001b[0;36mdecode\u001b[0;34m(input, errors)\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mdecode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'strict'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mcodecs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutf_8_decode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mIncrementalEncoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcodecs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIncrementalEncoder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;31mUnicodeDecodeError\u001b[0m: 'utf8' codec can't decode byte 0xae in position 0: invalid start byte"
       ]
      }
     ],
     "prompt_number": 121
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print re.sub(u\"\\\\u627e\",\"A\",u'\\u627e')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\u627e\n"
       ]
      }
     ],
     "prompt_number": 255
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tmp = fw.weibo_train_data.context[3]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 237
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tmp = tmp.decode('utf8')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 239
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tmp"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 242,
       "text": [
        "u'[doge]\\u627e\\u5de5\\u4f5c\\u256e(\\u256f\\u25bd\\u2570)\\u256d\\u2026\\u2026'"
       ]
      }
     ],
     "prompt_number": 242
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "len('ba')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 86,
       "text": [
        "2"
       ]
      }
     ],
     "prompt_number": 86
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "a=fw.pd.Series()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 261
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "a.append([1,2,3])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "TypeError",
       "evalue": "cannot concatenate a non-NDFrame object",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
        "\u001b[0;32m<ipython-input-262-bb81017f99b0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
        "\u001b[0;32m/Users/Pro/canopy/lib/python2.7/site-packages/pandas/core/series.pyc\u001b[0m in \u001b[0;36mappend\u001b[0;34m(self, to_append, verify_integrity)\u001b[0m\n\u001b[1;32m   1407\u001b[0m             \u001b[0mto_concat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mto_append\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1408\u001b[0m         return concat(to_concat, ignore_index=False,\n\u001b[0;32m-> 1409\u001b[0;31m                       verify_integrity=verify_integrity)\n\u001b[0m\u001b[1;32m   1410\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1411\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_binop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfill_value\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m/Users/Pro/canopy/lib/python2.7/site-packages/pandas/tools/merge.pyc\u001b[0m in \u001b[0;36mconcat\u001b[0;34m(objs, axis, join, join_axes, ignore_index, keys, levels, names, verify_integrity)\u001b[0m\n\u001b[1;32m    715\u001b[0m                        \u001b[0mignore_index\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mignore_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjoin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    716\u001b[0m                        \u001b[0mkeys\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlevels\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlevels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnames\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnames\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 717\u001b[0;31m                        verify_integrity=verify_integrity)\n\u001b[0m\u001b[1;32m    718\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    719\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m/Users/Pro/canopy/lib/python2.7/site-packages/pandas/tools/merge.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, objs, axis, join, join_axes, keys, levels, names, ignore_index, verify_integrity)\u001b[0m\n\u001b[1;32m    766\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mobjs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    767\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNDFrame\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 768\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"cannot concatenate a non-NDFrame object\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    769\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    770\u001b[0m             \u001b[0;31m# consolidate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;31mTypeError\u001b[0m: cannot concatenate a non-NDFrame object"
       ]
      }
     ],
     "prompt_number": 262
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "'\u25bd'.decode('utf8')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 69,
       "text": [
        "u'\\u25bd'"
       ]
      }
     ],
     "prompt_number": 69
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "'.'.decode('utf8')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 70,
       "text": [
        "u'.'"
       ]
      }
     ],
     "prompt_number": 70
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "'\u2026'.decode('utf8')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 72,
       "text": [
        "u'\\u2026'"
       ]
      }
     ],
     "prompt_number": 72
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "a[0]=[1,2,3]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "IndexError",
       "evalue": "index 0 is out of bounds for axis 0 with size 0",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
        "\u001b[0;32m<ipython-input-258-c47480f4cd6d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0ma\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
        "\u001b[0;32m/Users/Pro/canopy/lib/python2.7/site-packages/pandas/core/series.pyc\u001b[0m in \u001b[0;36m__setitem__\u001b[0;34m(self, key, value)\u001b[0m\n\u001b[1;32m    597\u001b[0m                     and not self.index.inferred_type == 'integer'):\n\u001b[1;32m    598\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 599\u001b[0;31m                 \u001b[0mvalues\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    600\u001b[0m                 \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    601\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mkey\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mEllipsis\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;31mIndexError\u001b[0m: index 0 is out of bounds for axis 0 with size 0"
       ]
      }
     ],
     "prompt_number": 258
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import weiboPredict"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "reload(weiboPredict.framework)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 11,
       "text": [
        "<module 'weiboPredict.framework' from 'weiboPredict/framework.py'>"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from weiboPredict import framework as fw"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "reload(framework)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "NameError",
       "evalue": "name 'framework' is not defined",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
        "\u001b[0;32m<ipython-input-283-3f9ab99c167d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mreload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mframework\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
        "\u001b[0;31mNameError\u001b[0m: name 'framework' is not defined"
       ]
      }
     ],
     "prompt_number": 283
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "fw.loadData()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tmp = fw.cleanText(fw.weibo_train_data.context[:5])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Building prefix dict from /Users/Pro/canopy/lib/python2.7/site-packages/jieba/dict.txt ...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "DEBUG:jieba:Building prefix dict from /Users/Pro/canopy/lib/python2.7/site-packages/jieba/dict.txt ...\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Loading model from cache /var/folders/rv/scpt0ywn277bqhjg160ms0d40000gp/T/jieba.cache\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "DEBUG:jieba:Loading model from cache /var/folders/rv/scpt0ywn277bqhjg160ms0d40000gp/T/jieba.cache\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Loading model cost 0.556 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "DEBUG:jieba:Loading model cost 0.556 seconds.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Prefix dict has been built succesfully.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "DEBUG:jieba:Prefix dict has been built succesfully.\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print '|'.join(tmp[1])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\u7231|\u5c0f\u7c73|\u624b\u673a|\u5c0f\u7c73|\u624b\u673a|\u5c4f\u5e55|\u592a\u7ed9\u529b|\u82f1\u5bf8|\u9ad8|\u8272\u5f69|\u9971\u548c\u5ea6|\u590f\u666e|JDI|\u5c4f\u5e55|x|\u5168\u9ad8\u6e05|\u5206\u8fa8\u7387|\u73a9\u6e38\u620f|\u9ad8\u6e05|\u89c6\u9891|\u723d\u7206|\u6708|\u65e5|\u4e2d\u5348|\u70b9|\u5f00\u653e|\u8d2d\u4e70|\u4ec5\u552e\u5143|\u5e0c\u671b|\u4e70|\u53f0|\u5c0f\u7c73|\u624b\u673a|\u5356|\u4ed9\u5251|\u9152\u5251\u4ed9|\u6d77|\u5148\u751f|V|\u5206\u4eab|\u514d\u8d39|\u9884\u7ea6|\u5730\u5740\n"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "t = fw.weibo_train_data"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 296
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print \"..\u54c8\u54c8\uff5d\uff1a:\".replace(string.punctuation,\"\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "..\u54c8\u54c8\uff5d\uff1a:\n"
       ]
      }
     ],
     "prompt_number": 97
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "t.head()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>uid</th>\n",
        "      <th>mid</th>\n",
        "      <th>time</th>\n",
        "      <th>forward_count</th>\n",
        "      <th>comment_count</th>\n",
        "      <th>like_count</th>\n",
        "      <th>context</th>\n",
        "      <th>a</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 07fc721342df1a4c1992560b582992f8</td>\n",
        "      <td> 5fefc2488f99aba5782baa43096b76dc</td>\n",
        "      <td> 2014-11-04</td>\n",
        "      <td> 0</td>\n",
        "      <td> 7</td>\n",
        "      <td> 2</td>\n",
        "      <td> [\u5410]@\u4eac\u4e1c \u7684\u7528\u6237\u4f53\u9a8c\u592a\u5dee\u4e86....\u7b2c\u4e00\u6b21\u5728\u4eac\u4e1c\u8d2d\u7269\uff0c\u4e70\u4e86\u51e0\u767e\u7684\u4e66....\u771f\u662f\u7ecf\u5386\u4e86\u7f51\u8d2d\u7684...</td>\n",
        "      <td> [\u5410, \u4eac\u4e1c, \u7528\u6237, \u4f53\u9a8c, \u592a\u5dee, ...., \u7b2c\u4e00\u6b21, \u4eac\u4e1c, \u8d2d\u7269, \u4e70, \u51e0\u767e, ...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 07fc721342df1a4c1992560b582992f8</td>\n",
        "      <td> 397d9abe2ebf9ba986fe5ab5a6139e11</td>\n",
        "      <td> 2014-11-04</td>\n",
        "      <td> 0</td>\n",
        "      <td> 3</td>\n",
        "      <td> 0</td>\n",
        "      <td> #\u6211\u7231\u5c0f\u7c73\u624b\u673a#\u56e0\u4e3a\u5c0f\u7c73\u624b\u673a4\u7684\u5c4f\u5e55\u592a\u7ed9\u529b\u4e86\uff0c5\u82f1\u5bf8\u7684\u9ad8\u8272\u5f69\u9971\u548c\u5ea6\u590f\u666e / JDI\u5c4f\u5e55\uff0c19...</td>\n",
        "      <td> [\u7231, \u5c0f\u7c73, \u624b\u673a, \u5c0f\u7c73, \u624b\u673a, \u5c4f\u5e55, \u592a\u7ed9\u529b, \u82f1\u5bf8, \u9ad8, \u8272\u5f69, \u9971\u548c\u5ea6, \u590f...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 07fc721342df1a4c1992560b582992f8</td>\n",
        "      <td> a2ae3959e9aa21a47993506e2609a4ec</td>\n",
        "      <td> 2014-11-08</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td>              [doge][\u55b5\u55b5]\u5929\u732b\u771f\u662f\u725b....\u5356\u5b9e\u9a8c\u8017\u6750\u7684\u90fd\u8dd1\u5929\u732b\u53bb\u5f00\u5e97\u4e86...</td>\n",
        "      <td> [doge, \u55b5, \u55b5, \u5929\u732b, \u771f\u662f, \u725b, ...., \u5356, \u5b9e\u9a8c, \u8017\u6750, \u8dd1, \u5929\u732b...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 07fc721342df1a4c1992560b582992f8</td>\n",
        "      <td> fcc63a086522040a271ba6d1fb604734</td>\n",
        "      <td> 2014-11-09</td>\n",
        "      <td> 0</td>\n",
        "      <td> 6</td>\n",
        "      <td> 0</td>\n",
        "      <td>                                [doge]\u627e\u5de5\u4f5c\u256e(\u256f\u25bd\u2570)\u256d\u2026\u2026</td>\n",
        "      <td>                      [doge, \u627e, \u5de5\u4f5c, \u256e, \u256f, \u25bd, \u2570, \u256d]</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> 07fc721342df1a4c1992560b582992f8</td>\n",
        "      <td> 6ab0f93ab59cbdaf25f4a023642971a6</td>\n",
        "      <td> 2014-11-11</td>\n",
        "      <td> 0</td>\n",
        "      <td> 5</td>\n",
        "      <td> 1</td>\n",
        "      <td>                                  [doge]\u6210\u529f\u62a2\u4e86\u53f0\u7c734\u7535\u4fe1~</td>\n",
        "      <td>                             [doge, \u6210\u529f, \u62a2, \u53f0\u7c73, \u7535\u4fe1]</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 298,
       "text": [
        "                                uid                               mid  \\\n",
        "0  07fc721342df1a4c1992560b582992f8  5fefc2488f99aba5782baa43096b76dc   \n",
        "1  07fc721342df1a4c1992560b582992f8  397d9abe2ebf9ba986fe5ab5a6139e11   \n",
        "2  07fc721342df1a4c1992560b582992f8  a2ae3959e9aa21a47993506e2609a4ec   \n",
        "3  07fc721342df1a4c1992560b582992f8  fcc63a086522040a271ba6d1fb604734   \n",
        "4  07fc721342df1a4c1992560b582992f8  6ab0f93ab59cbdaf25f4a023642971a6   \n",
        "\n",
        "         time  forward_count  comment_count  like_count  \\\n",
        "0  2014-11-04              0              7           2   \n",
        "1  2014-11-04              0              3           0   \n",
        "2  2014-11-08              0              0           0   \n",
        "3  2014-11-09              0              6           0   \n",
        "4  2014-11-11              0              5           1   \n",
        "\n",
        "                                             context  \\\n",
        "0  [\u5410]@\u4eac\u4e1c \u7684\u7528\u6237\u4f53\u9a8c\u592a\u5dee\u4e86....\u7b2c\u4e00\u6b21\u5728\u4eac\u4e1c\u8d2d\u7269\uff0c\u4e70\u4e86\u51e0\u767e\u7684\u4e66....\u771f\u662f\u7ecf\u5386\u4e86\u7f51\u8d2d\u7684...   \n",
        "1  #\u6211\u7231\u5c0f\u7c73\u624b\u673a#\u56e0\u4e3a\u5c0f\u7c73\u624b\u673a4\u7684\u5c4f\u5e55\u592a\u7ed9\u529b\u4e86\uff0c5\u82f1\u5bf8\u7684\u9ad8\u8272\u5f69\u9971\u548c\u5ea6\u590f\u666e / JDI\u5c4f\u5e55\uff0c19...   \n",
        "2               [doge][\u55b5\u55b5]\u5929\u732b\u771f\u662f\u725b....\u5356\u5b9e\u9a8c\u8017\u6750\u7684\u90fd\u8dd1\u5929\u732b\u53bb\u5f00\u5e97\u4e86...   \n",
        "3                                 [doge]\u627e\u5de5\u4f5c\u256e(\u256f\u25bd\u2570)\u256d\u2026\u2026   \n",
        "4                                   [doge]\u6210\u529f\u62a2\u4e86\u53f0\u7c734\u7535\u4fe1~   \n",
        "\n",
        "                                                   a  \n",
        "0  [\u5410, \u4eac\u4e1c, \u7528\u6237, \u4f53\u9a8c, \u592a\u5dee, ...., \u7b2c\u4e00\u6b21, \u4eac\u4e1c, \u8d2d\u7269, \u4e70, \u51e0\u767e, ...  \n",
        "1  [\u7231, \u5c0f\u7c73, \u624b\u673a, \u5c0f\u7c73, \u624b\u673a, \u5c4f\u5e55, \u592a\u7ed9\u529b, \u82f1\u5bf8, \u9ad8, \u8272\u5f69, \u9971\u548c\u5ea6, \u590f...  \n",
        "2  [doge, \u55b5, \u55b5, \u5929\u732b, \u771f\u662f, \u725b, ...., \u5356, \u5b9e\u9a8c, \u8017\u6750, \u8dd1, \u5929\u732b...  \n",
        "3                       [doge, \u627e, \u5de5\u4f5c, \u256e, \u256f, \u25bd, \u2570, \u256d]  \n",
        "4                              [doge, \u6210\u529f, \u62a2, \u53f0\u7c73, \u7535\u4fe1]  "
       ]
      }
     ],
     "prompt_number": 298
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tmp.to_csv('test.csv')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 299
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "clean_context = fw.cleanText(fw.weibo_train_data.context)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "10000/1626750\n",
        "20000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "30000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "40000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "50000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "60000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "70000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "80000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "90000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "100000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "110000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "120000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "130000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "140000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "150000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "160000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "170000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "180000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "190000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "200000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "210000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "220000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "230000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "240000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "250000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "260000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "270000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "280000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "290000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "300000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "310000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "320000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "330000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "340000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "350000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "360000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "370000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "380000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "390000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "400000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "410000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "420000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "430000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "440000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "450000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "460000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "470000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "480000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "490000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "500000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "510000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "520000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "530000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "540000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "550000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "560000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "570000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "580000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "590000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "600000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "610000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "620000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "630000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "640000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "650000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "660000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "670000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "680000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "690000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "700000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "710000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "720000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "730000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "740000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "750000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "760000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "770000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "780000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "790000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "800000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "810000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "820000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "830000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "840000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "850000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "860000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "870000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "880000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "890000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "900000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "910000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "920000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "930000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "940000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "950000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "960000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "970000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "980000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "990000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1000000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1010000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1020000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1030000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1040000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1050000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1060000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1070000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1080000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1090000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1100000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1110000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1120000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1130000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1140000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1150000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1160000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1170000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1180000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1190000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1200000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1210000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1220000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1230000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1240000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1250000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1260000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1270000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1280000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1290000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1300000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1310000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1320000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1330000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1340000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1350000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1360000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1370000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1380000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1390000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1400000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1410000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1420000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1430000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1440000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1450000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1460000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1470000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1480000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1490000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1500000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1510000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1520000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1530000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1540000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1550000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1560000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1570000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1580000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1590000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1600000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1610000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "1620000/1626750"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "predict_clean_text = fw.cleanText(fw.weibo_predict_data.context)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "10000/275331\n",
        "20000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "30000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "40000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "50000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "60000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "70000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "80000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "90000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "100000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "110000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "120000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "130000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "140000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "150000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "160000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "170000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "180000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "190000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "200000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "210000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "220000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "230000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "240000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "250000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "260000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "270000/275331"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "clean_context.head()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 8,
       "text": [
        "0    [\u5410, \u4eac\u4e1c, \u7528\u6237, \u4f53\u9a8c, \u592a\u5dee, ...., \u7b2c\u4e00\u6b21, \u4eac\u4e1c, \u8d2d\u7269, \u4e70, \u51e0\u767e, ...\n",
        "1    [\u7231, \u5c0f\u7c73, \u624b\u673a, \u5c0f\u7c73, \u624b\u673a, \u5c4f\u5e55, \u592a\u7ed9\u529b, \u82f1\u5bf8, \u9ad8, \u8272\u5f69, \u9971\u548c\u5ea6, \u590f...\n",
        "2    [doge, \u55b5, \u55b5, \u5929\u732b, \u771f\u662f, \u725b, ...., \u5356, \u5b9e\u9a8c, \u8017\u6750, \u8dd1, \u5929\u732b...\n",
        "3                         [doge, \u627e, \u5de5\u4f5c, \u256e, \u256f, \u25bd, \u2570, \u256d]\n",
        "4                                [doge, \u6210\u529f, \u62a2, \u53f0\u7c73, \u7535\u4fe1]\n",
        "dtype: object"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "clean_context.to_csv('data/train_context_clean.csv')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "predict_clean_text.to_csv('data/predict_context_clean.csv')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}